package com.yuanjiu.service.w1919;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.yuanjiu.common.util.Global;
import com.yuanjiu.common.util.HtpClientUtils;
import com.yuanjiu.dao.w1919.ItemDao;
import com.yuanjiu.dao.yuanjiu.DataMergeDao;
import com.yuanjiu.pojo.w1919.W1919Comment;
import com.yuanjiu.pojo.w1919.w1919SpecObject;
import com.yuanjiu.pojo.yuanjiu.CategoryIdName;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.selector.Selectable;

@Service
public class W19Data {
	
	public static int itemcount =0;

	@Autowired
	private DataCapture dataCapture;

	@Autowired
	private DataPersistence dataOutput;

	// 存储类别和url映射
	private Map<String, LinkedBlockingQueue<String>> container = new HashMap<String, LinkedBlockingQueue<String>>();

	public void process(String... param) {
		container.put("白酒", new LinkedBlockingQueue<String>());
		container.put("葡萄酒", new LinkedBlockingQueue<String>());
		container.put("洋酒", new LinkedBlockingQueue<String>());
		container.put("啤酒", new LinkedBlockingQueue<String>());
		container.put("黄酒", new LinkedBlockingQueue<String>());
		container.put("果酒", new LinkedBlockingQueue<String>());
		container.put("其他", new LinkedBlockingQueue<String>());
		dataCapture.setContainer(container);
		dataCapture.setUpIdNames();

		// 抓取商品
		Spider.create(dataCapture)
				.addUrl(param)
				.addPipeline(dataOutput).thread(6).run();
	}

}

@Service
class DataCapture implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setRetrySleepTime(3000);

//	@Autowired
//	@Qualifier("w1919Bean")
//	private CategoryDao dao;
	
	@Autowired
	private DataMergeDao dataMergeDao;
	
	private Map<String, LinkedBlockingQueue<String>> container;
	private List<CategoryIdName> idNames;

	public DataCapture() {

	}
	
	public DataCapture(Map<String, LinkedBlockingQueue<String>> container) {
		this.container = container;
		try {
			
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public void setContainer(Map<String, LinkedBlockingQueue<String>> container) {
		this.container = container;
	}

	public List<CategoryIdName> getIdNames() {
		return idNames;
	}

	public void setUpIdNames() {
		try {
			this.idNames= dataMergeDao.getCategoryName();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	@Override
	public void process(Page page) {
		Html htmlContent = page.getHtml();

		// 列表页，提取下一页加入队列
		if (page.getUrl().regex(".*/search.html(.*)?cat=.*").match()) {
			String urlBase = page.getUrl().get().split("\\?")[0];
			// 获取下一页, 根据页面下一页点击事件searchPage js函数内容，最大保证可行性
			//sort参数保持默认
			urlBase += "?sort=DEFAULT_SORT&size=16";
			
			// vendorId，1是自营商品， 空字符串表示不限定自营， 此处抓取全部
			String vendorId="";
			urlBase += "&vendorId="+vendorId;
			
			//关键词
			String kw = htmlContent.xpath("//*[@id=showkeyword]/@showkeyword").get();
			if (kw != null && kw != "") {
				urlBase += "&kw=" + kw;
			}
			// cat
			String catid = htmlContent.xpath("//*[@id=categoryId]/@categoryId").get();
			String catVaue = htmlContent.xpath("//*[@id=categoryId]/@name").get();
			if (catid != null && !catid.isEmpty() && catVaue != null && !catVaue.isEmpty()) {
				urlBase += "&cat=" + catid + "_" + catVaue;
			}
			// type
			String typeContent = htmlContent.xpath("//*[@id=vendorType]/text()").get();
			if (typeContent != null && !typeContent.isEmpty()) {
				urlBase += "&type=" + typeContent;
			} else {
				urlBase += "&type="+vendorId;
			}
			
			//fields参数设置筛选条件，忽略
				
			// 下一页
			Selectable np = htmlContent.css("body > div.g-cont > div > div.m-ct > div.m-pdt > div.m-pagin > ul > li.pt-item.pt-next > a");
			if (np != null) {
				String npNum = np.xpath("/a/@onclick").get();
				if(StringUtils.isNotEmpty(npNum)){
					npNum=npNum.split("\\(")[1].split("\\)")[0];
					urlBase += "&page=" + npNum;
					// 加入队列,优先级80，数字越大优先级越高
					page.addTargetRequest(new Request(urlBase).setPriority(80));
				}
			}

			// 获取商品连接加入队列，同时加入container保存类别-商品映射关系（页面类别信息显示不全），下同
			// 获取类别信息，白酒...
			String catName = htmlContent.xpath("//*[@id=categoryId]/@name").get();
			// 获取list加入Url
			if (catName == null) {
				catName = "其他";
			}
			LinkedBlockingQueue<String> listContetnt = container.get(catName);
			List<Selectable> list = htmlContent.xpath("//*[@id=productList]/li").nodes();
			for (Selectable tmp : list) {
				String link = tmp.xpath("/li/div[1]/a/@href").get();
				listContetnt.add(link);
				page.addTargetRequest(new Request(link).setPriority(100));
			}
			// 分类页面结束
			String utl = "";
			for(String tmp : listContetnt){
				utl += tmp +"\r\n";
			}
			System.out.println(utl);
		}

		// 详情页
		if (page.getUrl().regex(".*www.1919.cn/p/[0-9]+.*").match()) {
			// 获取商品详情
			// id
			w1919SpecObject item = new w1919SpecObject();
			String urlsuffix = page.getUrl().get().split("/")[4];
			String id = urlsuffix.substring(0, urlsuffix.indexOf("."));
			item.setId(id);

			// 查找一级分类
			String categoryOne = null;
			String categoryOneId = "0";
			for (Map.Entry<String, LinkedBlockingQueue<String>> tmp : container.entrySet()) {
				Iterator<String> list = tmp.getValue().iterator();
				String url=page.getUrl().get();
				boolean flag=false;
				while(list.hasNext()){
					String next = list.next();
					if(StringUtils.equals(next, url)){
						categoryOne = tmp.getKey();
						flag=true;
						break;
					}
				}
				if(flag){
					break;
				}
			}
			if (categoryOne == null) {
				categoryOne = "其他";
			}
			for (CategoryIdName tmp2 : idNames) {
				if (StringUtils.equals(tmp2.getCat(), categoryOne)) {
					categoryOneId = tmp2.getId() != null ?tmp2.getId().toString():"";
					break;
				}
			}
			// 一级分类id不存在则直接返回
			if (categoryOneId == null || "".equals(categoryOneId)) {
				return;
			}

			// 获取品牌名称
			String brandName = htmlContent.xpath("/html/body/div[5]/div/p/span[3]/text()").get();
			List<String> brandList = null;
			try {
				brandList = dataMergeDao.getCategoriesByParent(Integer.valueOf(categoryOneId));
				if (brandList == null) {
					return;
				} else {
					if (!brandList.contains(brandName)) {
						brandName = "其他";
					}
				}
				String catid = dataMergeDao.getCategoryIdByName(brandName,categoryOneId);  
				item.setCategory(catid);
			} catch (Exception e) {
				e.printStackTrace();
			}

			// 获取title
			String title = htmlContent.xpath("/html/body/div[6]/div[2]/div[1]/h1/text()").get();
			item.setTitle(title);

			// 建议价格 ￥2.0
			String valuesuggest = htmlContent.xpath("/html/body/div[6]/div[2]/div[2]/div[1]/span[2]/text()").get();
			if (valuesuggest != null && !"".equals(valuesuggest)) {
				item.setValuesuggest(valuesuggest.substring(1));
			}

			// 售价 ￥45
			String valueSell = htmlContent.xpath("/html/body/div[6]/div[2]/div[2]/div[2]/div/em/text()").get();
			if (valueSell != null && !"".equals(valueSell)) {
				item.setValuesell(valueSell);
			}

			// 库存
			String storage = htmlContent.xpath("/html/body/div[6]/div[2]/div[3]/div[1]/div[2]/a/text()").get();
			if (storage != null && !"".equals(storage)) {
				item.setStorage(storage);
			}

			// 店铺名称
			String shopname = htmlContent.xpath("/html/body/div[8]/div/div[2]/div/div[1]/div/div/div[1]/div[2]/h1/text()").get();
			if(StringUtils.isNotEmpty(shopname)){
				if (shopname.contains("自营")) {
					shopname += "-自营";
					item.setShopname(shopname);
				}else{
					item.setShopname(shopname);
				}
			}

			// 下载图片
			List<Selectable> imageList = htmlContent.xpath("//*[@id='imageMenu']/ul/li").nodes();
			String pic = "";
			for (Selectable tmp : imageList) {
				String im1 = tmp.xpath("/li/img/@src").get();
				if (StringUtils.isNotEmpty(im1)) {
					// 下载图片
//					String fileName = im1.split("\\?")[0];
//					String fileName2 = fileName.substring(fileName.lastIndexOf("/") + 1)+".jpg";
//					try {
//						downloadImage(im1, fileName2, Global.getW1919ImgPath());
//					} catch (Exception e) {
//						e.printStackTrace();
//					}
//
//					if ("".equals(pic)) {
//						pic = fileName2;
//					} else {
//						pic += "," + fileName2;
//					}
					
					if(StringUtils.isEmpty(pic)){
						pic  = im1;
					}else{
						pic+=","+im1;
					}
				}
			}
			item.setPic(pic);

			// 详情
			List<Selectable> paramList = htmlContent.xpath("//*[@id='com-intro']/div[2]/div[1]/div").nodes();
			for (Selectable tmp : paramList) {
				String paramName = tmp.xpath("/div/span[@class='cont-title']/text()").get();
				String paramValue = tmp.xpath("/div/span[@class='cont-cont']/text()").get();
				if(paramName==null || paramValue==null){
					continue;
				}
				// 商品编号
				if (paramName.contains("编码")) {
					item.setGno(paramValue);
				}
				// 香型
				if (paramName.contains("香型")) {
					item.setFlavor(paramValue);
				}
				// 产地
				if (paramName.contains("产地")) {
					item.setProduce(paramValue);
				}
				// 净含量
				if (paramName.contains("净含量")) {
					item.setNet(paramValue);
				}
				// 酒精度
				if (paramName.contains("酒精度")) {
					item.setDegree(paramValue);
				}
			}
			
			//若参数中没有酒精度和容量，则从标题中获取酒精度与容量
			//处理30-44度，提取具体数字
			if(item.getDegree()==null||"".equals(item.getDegree()) || item.getDegree().contains("-")){
				String tmpTitle=item.getTitle();
				if(StringUtils.isNotEmpty(tmpTitle)){
					int end=tmpTitle.indexOf("°");
					if(end==-1){
						end=tmpTitle.indexOf("度");
					}
					if(end!=-1){
						tmpTitle=tmpTitle.substring(0, end);
						 Matcher matcher = Pattern.compile("[0-9]+$").matcher(tmpTitle);
						 if(matcher.find()){
						    String deg=matcher.group();
						    if(StringUtils.isNotEmpty(deg)){
						    	item.setDegree(deg);
						    }
						 }
					}
				}
			}
			//同上
			if(item.getNet()==null||"".equals(item.getNet()) || item.getNet().contains("-")){
				String tmpTitle=item.getTitle();
				if(StringUtils.isNotEmpty(tmpTitle)){
					int end=tmpTitle.indexOf("ml");
					if(end==-1){
						end=tmpTitle.indexOf("ML");
					}
					if(end==-1){
						end=tmpTitle.indexOf("Ml");
					}
					if(end==-1){
						end=tmpTitle.indexOf("mL");
					}
					if(end!=-1){
						tmpTitle=tmpTitle.substring(0, end);
						Matcher matcher = Pattern.compile("[0-9]+$").matcher(tmpTitle);
						if(matcher.find()){
						    String net=matcher.group();
						    item.setNet(net);
						}
					}
				}
			}
			// 抓取时间
			item.setGrabtime(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(Calendar.getInstance().getTime()));
			// url
			item.setUrl(page.getUrl().get());
			page.putField("item", item);
			
			//评论抓取
			String productCode="";
			String productId="";
			Integer pageNum=1;
			String vendorId="";
			boolean flag=false;
			String commentParam=htmlContent.css("head > script:nth-child(7)").get();
			if(StringUtils.isNotEmpty(commentParam)){
				String params[]=commentParam.split(",");
				for(String tmp:params){
					if(tmp.contains("productCode")){
						productCode=tmp.substring(tmp.indexOf("'")+1, tmp.lastIndexOf("'"));
					}else if(tmp.contains("productId")){
						productId=tmp.substring(tmp.indexOf("'")+1, tmp.lastIndexOf("'"));
					}else if(tmp.contains("vendorId")){
						vendorId=tmp.substring(tmp.indexOf("'")+1, tmp.lastIndexOf("'"));
					}
				}
			}
			//超过20条https://www.1919.cn/product/commentData?productCode=1000095&productId=346840940184473605&page=1&vendorId=346833411299741697,page从1开始
			//7条评论 https://www.1919.cn/product/commentData?productCode=1005267&productId=346840940238999559&page=1&vendorId=346833410985168897
			//没有评论https://www.1919.cn/product/commentData?productCode=1013195&productId=366329680322105344&page=1&vendorId=346840538491785216
			
//			String commentUrl=Global.getW19Url()+"product/commentData?productCode="+productCode+"&productId="+productId+"&page="+page+"&vendorId="+vendorId;
			//新建一个Page对象存储评论内容
			Page page2=new Page();
			List<W1919Comment> commentList=new ArrayList<W1919Comment>();
			int commentCount =1;
			for(;pageNum<=2;pageNum++){
				String commentUrl=Global.getW19Url()+"product/commentData?productCode="+productCode+"&productId="+productId+"&page="+pageNum+"&vendorId="+vendorId;
				String result=HtpClientUtils.doGet(commentUrl);
				page2.setRequest(new Request(commentUrl));
				page2.setUrl(new PlainText(commentUrl));
				page2.setRawText(result);
				Html commentHtml = page2.getHtml();
				String resStep1=commentHtml.xpath("/html/body/div[2]/div[1]/ul").get();
				if(resStep1==null || "".equals(resStep1)){
					flag=true;
				}else{
					List<Selectable> list = commentHtml.xpath("/html/body/div[2]/div[1]/ul/li").nodes();
					if(!list.isEmpty()){
						for(Selectable tmp:list){
							String commentContent=tmp.xpath("/li/div[2]/div//span[2]/text()").get();
							String commentTime=tmp.xpath("/li/div[2]/div/div[2]/text()").get();
							W1919Comment w1919Comment=new W1919Comment();
							w1919Comment.setId(item.getId()+"-"+(commentCount+=1));
							w1919Comment.setCom(commentContent);
							w1919Comment.setComment_time(commentTime);
							w1919Comment.setGid(id);
							w1919Comment.setGrabTime(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(Calendar.getInstance().getTime()));
							commentList.add(w1919Comment);
						}
						//判断有没有下一页
						String totalComment = commentHtml.xpath("//*[@id='commentTab']/li/a/span/text()").get();
						if(StringUtils.isNotEmpty(totalComment))
							try{
								if(Integer.valueOf(totalComment) < 10){
									break;
								}
							}catch (Exception e) {
							}
						}
					}
			}
			if(!flag){
				page.putField("comment", commentList);
			}

		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	public void downloadImage(String urlString, String filename, String savePath) throws Exception {
		// 构造URL
		URL url = new URL(urlString);
		// 打开连接
		URLConnection con = url.openConnection();
		// 设置请求超时为5s
		con.setConnectTimeout(5 * 1000);
		// 输入流
		InputStream is = con.getInputStream();

		// 1K的数据缓冲
		byte[] bs = new byte[1024];
		// 读取到的数据长度
		int len;
		// 输出的文件流
		File sf = new File(savePath);
		if (!sf.exists()) {
			sf.mkdirs();
		}
		OutputStream os = new FileOutputStream(sf.getPath() + "\\" + filename);
		// 开始读取
		while ((len = is.read(bs)) != -1) {
			os.write(bs, 0, len);
		}
		// 完毕，关闭所有链接
		os.close();
		is.close();
	}
}

@Service
class DataPersistence implements Pipeline {
	@Autowired
	private ItemDao dao;

	@Override
	public void process(ResultItems resultItems, Task task) {
		w1919SpecObject res = resultItems.get("item");
		if (res != null) {
			dao.insert(res);
			W19Data.itemcount += 1;
		}
		
		List<W1919Comment> comments=resultItems.get("comment");
		if(CollectionUtils.isNotEmpty(comments)){
			//评论id自动生成，每次清空
			//不再自动生成
//			dao.clearComments(res); 
			dao.saveComments(comments);
		}
	}

}